# ============================================================================
# 民宿价格预测 - 可分段运行脚本
# 目标：将MSE降低到30以下
# ============================================================================

# %% [段落1] 导入必要的库和模块
print("=" * 60)
print("段落1: 导入必要的库和模块")
print("=" * 60)

import pandas as pd
import numpy as np
import lightgbm as lgb
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import KFold, train_test_split
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.metrics import mean_squared_error
import warnings
warnings.filterwarnings('ignore')

print("✅ 所有必要的库已成功导入")

# %% [段落2] 数据加载和基本信息查看
print("\n" + "=" * 60)
print("段落2: 数据加载和基本信息查看")
print("=" * 60)

# 加载数据
train = pd.read_csv('训练集.csv')
test = pd.read_csv('测试集.csv')

print(f"训练集形状: {train.shape}")
print(f"测试集形状: {test.shape}")
print(f"训练集列名: {list(train.columns)}")
print(f"\n价格统计信息:")
print(train['价格'].describe())

# 查看缺失值
print(f"\n训练集缺失值:")
missing_train = train.isnull().sum()
print(missing_train[missing_train > 0])

print("✅ 数据加载完成，基本信息已显示")

# %% [段落3] 高级特征工程
print("\n" + "=" * 60)
print("段落3: 高级特征工程")
print("=" * 60)

def advanced_feature_engineering(df):
    """
    高级特征工程函数
    """
    print(f"原始数据形状: {df.shape}")
    
    # 1. 便利设施特征
    df['便利设施数量'] = df['便利设施'].apply(lambda x: len(x.lstrip('{').rstrip('}').split(',')))
    
    # 2. 处理异常价格值（仅对训练集）
    if '价格' in df.columns:
        original_len = len(df)
        df = df[df['价格'] > 0].copy()  # 移除价格为0的异常值
        print(f"移除了 {original_len - len(df)} 个异常价格记录")
    
    # 3. 房间相关特征
    df['房间总数'] = df['卧室数量'] + df['洗手间数量']
    df['人均房间数'] = df['房间总数'] / (df['容纳人数'] + 1)  # 避免除零
    df['床位密度'] = df['床的数量'] / (df['容纳人数'] + 1)
    df['房间利用率'] = df['床的数量'] / (df['房间总数'] + 1)
    
    # 4. 地理位置特征
    df['经纬度距离'] = np.sqrt(df['维度']**2 + df['经度']**2)
    df['维度_标准化'] = (df['维度'] - df['维度'].mean()) / df['维度'].std()
    df['经度_标准化'] = (df['经度'] - df['经度'].mean()) / df['经度'].std()
    
    # 5. 评论相关特征
    df['有评论'] = (df['评论个数'] > 0).astype(int)
    df['评论密度'] = df['评论个数'] / (df['容纳人数'] + 1)
    df['评论强度'] = df['评论个数'] * df['民宿评分'].fillna(0) / 100
    
    # 6. 房主特征
    df['房主是否有头像'] = (df['房主是否有个人资料图片'].astype(str) == 't').astype(int)
    df['房主是否验证'] = (df['房主身份是否验证'].astype(str) == 't').astype(int)
    df['房主完整度'] = df['房主是否有头像'] + df['房主是否验证']
    
    # 7. 时间特征处理
    date_cols = ['首次评论日期', '最近评论日期', '何时成为房主']
    for col in date_cols:
        if col in df.columns:
            df[col] = pd.to_datetime(df[col], errors='coerce')
            df[f'{col}_年'] = df[col].dt.year
            df[f'{col}_月'] = df[col].dt.month
            df[f'{col}_季度'] = df[col].dt.quarter
            df[f'{col}_是否缺失'] = df[col].isnull().astype(int)
    
    # 8. 房主回复率处理
    if '房主回复率' in df.columns:
        df['房主回复率'] = df['房主回复率'].astype(str).str.replace('%', '').replace('', '0')
        df['房主回复率'] = pd.to_numeric(df['房主回复率'], errors='coerce').fillna(0)
        df['房主回复率_分类'] = pd.cut(df['房主回复率'], bins=[0, 50, 80, 100], labels=[0, 1, 2])
    
    # 9. 价格相关特征（仅对训练集）
    if '价格' in df.columns:
        df['价格_对数'] = np.log1p(df['价格'])
        df['价格_分类'] = pd.qcut(df['价格'], q=5, labels=[0, 1, 2, 3, 4])
    
    # 10. 组合特征
    df['容纳_便利设施_比'] = df['容纳人数'] / (df['便利设施数量'] + 1)
    df['评分_评论_交互'] = df['民宿评分'].fillna(0) * df['评论个数']
    
    print(f"特征工程后数据形状: {df.shape}")
    return df

# 应用特征工程
train_processed = advanced_feature_engineering(train.copy())
test_processed = advanced_feature_engineering(test.copy())

print("✅ 高级特征工程完成")

# %% [段落4] 数据预处理和编码
print("\n" + "=" * 60)
print("段落4: 数据预处理和编码")
print("=" * 60)

# 定义不需要的特征
no_features = ['数据ID', '价格', '便利设施', '首次评论日期', '最近评论日期', '何时成为房主', '价格_对数', '价格_分类']

# 合并数据进行统一编码
data_combined = pd.concat([train_processed, test_processed], axis=0)

# 对类别变量进行标签编码
categorical_cols = train_processed.select_dtypes(include=['object']).columns
print(f"需要编码的类别变量: {list(categorical_cols)}")

for col in categorical_cols:
    if col not in no_features:
        lb = LabelEncoder()
        lb.fit(data_combined[col].astype(str))
        train_processed[col] = lb.transform(train_processed[col].astype(str))
        test_processed[col] = lb.transform(test_processed[col].astype(str))

# 获取最终特征列
features = [col for col in train_processed.columns if col not in no_features]
print(f"最终使用的特征数量: {len(features)}")
print(f"特征列表: {features[:10]}...")  # 显示前10个特征

# 准备训练数据
X = train_processed[features]
y = train_processed['价格']
X_test = test_processed[features]

print("✅ 数据预处理和编码完成")

# %% [段落5] 数据标准化
print("\n" + "=" * 60)
print("段落5: 数据标准化")
print("=" * 60)

# 标准化数值特征
scaler = StandardScaler()
numeric_features = X.select_dtypes(include=[np.number]).columns
print(f"需要标准化的数值特征数量: {len(numeric_features)}")

X_scaled = X.copy()
X_test_scaled = X_test.copy()

X_scaled[numeric_features] = scaler.fit_transform(X[numeric_features])
X_test_scaled[numeric_features] = scaler.transform(X_test[numeric_features])

print("✅ 数据标准化完成")

# %% [段落6] 模型参数优化和训练
print("\n" + "=" * 60)
print("段落6: 模型参数优化和训练")
print("=" * 60)

# 分割训练和验证集
X_train, X_val, y_train, y_val = train_test_split(X_scaled, y, test_size=0.2, random_state=42)

print(f"训练集大小: {X_train.shape}")
print(f"验证集大小: {X_val.shape}")

# 优化的LightGBM参数
optimized_params = {
    'learning_rate': 0.03,
    'boosting_type': 'gbdt',
    'objective': 'regression',
    'metric': 'rmse',
    'feature_fraction': 0.9,
    'bagging_fraction': 0.9,
    'bagging_freq': 3,
    'num_leaves': 300,
    'verbose': -1,
    'max_depth': 6,
    'seed': 2019,
    'reg_alpha': 0.3,
    'reg_lambda': 0.3,
    'min_child_samples': 20,
    'subsample_for_bin': 200000,
    'min_split_gain': 0.1,
}

print("开始训练优化模型...")
model = lgb.LGBMRegressor(**optimized_params, n_estimators=3000)
model.fit(X_train, y_train, 
          eval_set=[(X_val, y_val)], 
          eval_metric='rmse',
          callbacks=[lgb.log_evaluation(500), lgb.early_stopping(150)])

# 预测和评估
y_pred = model.predict(X_val)
mse_score = mean_squared_error(y_val, y_pred)
rmse_score = np.sqrt(mse_score)

print(f"\n模型性能评估:")
print(f"MSE分数: {mse_score:.2f}")
print(f"RMSE分数: {rmse_score:.2f}")

if mse_score < 30:
    print("🎉 成功！MSE分数已降低到30以下")
else:
    print("⚠️  MSE分数仍需进一步优化")

print("✅ 模型训练和评估完成")

# %% [段落7] 5折交叉验证
print("\n" + "=" * 60)
print("段落7: 5折交叉验证")
print("=" * 60)

n_fold = 5
folds = KFold(n_splits=n_fold, shuffle=True, random_state=1314)

oof_predictions = np.zeros(len(X_scaled))
test_predictions = np.zeros(len(X_test_scaled))
cv_scores = []

print("开始5折交叉验证...")

for fold_n, (train_index, valid_index) in enumerate(folds.split(X_scaled)):
    print(f"\n训练第 {fold_n + 1} 折...")
    
    X_train_fold = X_scaled.iloc[train_index]
    X_valid_fold = X_scaled.iloc[valid_index]
    y_train_fold = y.iloc[train_index]
    y_valid_fold = y.iloc[valid_index]
    
    # 训练模型
    fold_model = lgb.LGBMRegressor(**optimized_params, n_estimators=3000)
    fold_model.fit(X_train_fold, y_train_fold,
                   eval_set=[(X_valid_fold, y_valid_fold)],
                   eval_metric='rmse',
                   callbacks=[lgb.early_stopping(150)])
    
    # 预测
    y_pred_valid = fold_model.predict(X_valid_fold)
    y_pred_test = fold_model.predict(X_test_scaled)
    
    # 保存预测结果
    oof_predictions[valid_index] = y_pred_valid
    test_predictions += y_pred_test / n_fold
    
    # 计算分数
    fold_score = mean_squared_error(y_valid_fold, y_pred_valid)
    cv_scores.append(fold_score)
    print(f"第 {fold_n + 1} 折 MSE: {fold_score:.2f}")

# 计算最终交叉验证分数
final_cv_score = mean_squared_error(y, oof_predictions)
print(f"\n交叉验证结果:")
print(f"各折MSE分数: {[f'{score:.2f}' for score in cv_scores]}")
print(f"平均MSE分数: {np.mean(cv_scores):.2f} ± {np.std(cv_scores):.2f}")
print(f"最终CV MSE分数: {final_cv_score:.2f}")

if final_cv_score < 30:
    print("🎉 交叉验证成功！MSE分数已降低到30以下")
else:
    print("⚠️  交叉验证MSE分数仍需进一步优化")

print("✅ 5折交叉验证完成")

# %% [段落8] 生成提交结果
print("\n" + "=" * 60)
print("段落8: 生成提交结果")
print("=" * 60)

# 生成最终预测结果
test_processed['价格'] = test_predictions

# 保存结果
submission_filename = f'submission_mse_{final_cv_score:.2f}.csv'
test_processed[['数据ID', '价格']].to_csv(submission_filename, index=False)

print(f"预测结果已保存到: {submission_filename}")
print(f"提交文件前5行:")
print(test_processed[['数据ID', '价格']].head())

# 显示预测价格的统计信息
print(f"\n预测价格统计:")
print(test_processed['价格'].describe())

print("✅ 提交结果生成完成")

# %% [段落9] 特征重要性分析
print("\n" + "=" * 60)
print("段落9: 特征重要性分析")
print("=" * 60)

# 获取特征重要性
feature_importance = pd.DataFrame({
    'feature': features,
    'importance': model.feature_importances_
}).sort_values('importance', ascending=False)

print("前15个最重要特征:")
print(feature_importance.head(15))

# 可视化特征重要性（前10个）
plt.figure(figsize=(10, 6))
top_features = feature_importance.head(10)
plt.barh(range(len(top_features)), top_features['importance'])
plt.yticks(range(len(top_features)), top_features['feature'])
plt.xlabel('特征重要性')
plt.title('前10个最重要特征')
plt.gca().invert_yaxis()
plt.tight_layout()
plt.savefig('feature_importance.png', dpi=300, bbox_inches='tight')
plt.show()

print("✅ 特征重要性分析完成")

# %% [段落10] 总结报告
print("\n" + "=" * 60)
print("段落10: 总结报告")
print("=" * 60)

print("🏆 民宿价格预测项目完成总结:")
print(f"📊 最终MSE分数: {final_cv_score:.2f}")
print(f"📈 目标达成: {'✅ 是' if final_cv_score < 30 else '❌ 否'} (目标 < 30)")
print(f"🔧 使用特征数量: {len(features)}")
print(f"📁 提交文件: {submission_filename}")
print(f"🎯 模型类型: LightGBM回归器")
print(f"🔄 验证方法: 5折交叉验证")

print("\n主要优化策略:")
print("1. ✅ 高级特征工程 - 创建了房间利用率、评论强度等组合特征")
print("2. ✅ 数据清洗 - 移除了异常价格值")
print("3. ✅ 参数调优 - 降低学习率，增加正则化")
print("4. ✅ 数据标准化 - 对数值特征进行标准化")
print("5. ✅ 交叉验证 - 使用5折交叉验证确保模型稳定性")

print("\n" + "=" * 60)
print("🎉 所有段落执行完成！")
print("=" * 60)